"""

.. module:: node
.. moduleauthor:: @author: 

The purpose of this module is to control the node installation and boot. Note
that it is a goal of this module that it NOT include any other module except
it does have the overhead of requiring some code duplication with other modules
that would ordinarily be imported into this.

Notes about running the node. There are two ways to run the node - either 
as a service or from a shell. When run as a service the node must register 
with Windows Services so Windows knows the node has started. When running
from a shell we don't need to register with Windows. Therefore we need 
something on the command line that tells us which startup we're 
performing. Running as a service is the most likely scenario so we'll 
require the "start" command when being started from a shell.

*Constants*:

.. data:: DEFAULT_DISTRIBUTION_DATASTORE_FILENAME

    DEFAULT_DISTRIBUTION_DATASTORE_FILENAME is the default name of the 
    distribution datastore.
    
.. data:: DEFAULT_DOWNLOAD_SITE_URL

    DEFAULT_DOWNLOAD_SITE_URL is the default URL of the download site.

.. data:: DEFAULT_DOWNLOAD_SITE_DISTRIBUTION_DATASTORE_URL

    DEFAULT_DOWNLOAD_SITE_DISTRIBUTION_DATASTORE_URL is the default URL of the 
    distribution datastore.
    
.. data:: CENTRAL_NODE_URL_TEST

    CENTRAL_NODE_URL_TEST is the default URL of the central node (test).

.. data:: CENTRAL_NODE_URL_PRODUCTION

    CENTRAL_NODE_URL_PRODUCTION is the default URL of the central node 
    (production).

.. data:: DEFAULT_PROXY_PORT

    DEFAULT_PROXY_PORT is the default port number of a proxy server
    
.. data:: TEST_SITE_URL

    TEST_SITE_URL is the URL we request to test connectivity through a proxy.

.. data:: RESOURCE_KEY_URI_PACKAGENAME

    RESOURCE_KEY_URI_PACKAGENAME is the Resource name of a python package 
    within the datastore.

.. data:: RESOURCE_KEY_URI_MODULENAME

    RESOURCE_KEY_URI_MODULENAME is the Resource name of a python module within 
    the datastore.

.. seealso:: 
"""

# Imports
#
import base64
import imp
import logging
from optparse import OptionParser
import os
# import signal
# import socket
import SocketServer
import sqlite3
import string
import sys
import threading
import subprocess
import urllib2
import urlparse
import uuid
import xml.etree.ElementTree
import zlib

if sys.platform == 'win32':
    import win32evtlog
    from logging.handlers import NTEventLogHandler
elif sys.platform == 'darwin': # OS/X
    pass
else: # assume Linux
    pass

# Constants
#
DEFAULT_DISTRIBUTION_DATASTORE_FILENAME = "temp.datastore"
DEFAULT_DOWNLOAD_SITE_URL = "http://yarra.googlecode.com/files"
DEFAULT_ADMIN_SITE_URL = "http://yarra.googlecode.com/"
DEFAULT_NODE_IP = "localhost"
DEFAULT_PORT_RANGE_START = 8900
DEFAULT_PORT_RANGE_END = 8909
DEFAULT_START_AS_WINDOWS_SERVICE = False
DEFAULT_DOWNLOAD_SITE_DISTRIBUTION_DATASTORE_URL = \
                ''.join([DEFAULT_DOWNLOAD_SITE_URL, 
                "/", DEFAULT_DISTRIBUTION_DATASTORE_FILENAME])
CENTRAL_NODE_URL_TEST = "localhost"
CENTRAL_NODE_URL_PRODUCTION = "localhost"
DEFAULT_PROXY_PORT = 80
TEST_SITE_URL = "http://www.google.com"

TEN_MEGABYTES_IN_BYTES = 1024 * 1024 * 10
LOG_FILE_NAME = "LOG"

# These are duplicated from datastore.resource.Resource because we don't 
# want to import Resource.
#RESOURCE_KEY_URI_PACKAGENAME = "term:base/build/package"
#RESOURCE_KEY_URI_MODULENAME = "term:base/build/module"
RESOURCE_KEY_URI_PACKAGE_MODULE = \
    "http://node.internal/data_dictionary/base/build/package_module"
RESOURCE_KEY_URI_FILE = \
    "http://node.internal/data_dictionary/base/build/file"    

# Classes
#
class Console(object):
    """
    The purpose of this class is to handle I/O from the console.

    **Constants**:
    """
    
    # Class constants
    #
    
    def __init__(self):
        # Constructor.
        #
        
        pass
    
    @staticmethod
    def display(msg):
        """
        This function displays the argument message on the console.
        
        :param msg: String - message to display to user.
        """
        
        print msg
    
    @staticmethod
    def getString(msg, default="", prompt="> ", mandatory=True):
        """
        This function displays the argument message on the console,
        displays a prompt on the console, waits for user input, and 
        returns the input, or 'default' if no input. A String value 
        is always returned. If 'mandatory' argument is True (default) 
        then user will be re-prompted if no value entered. However, if
        'mandatory' is True and 'default' is supplied then 'default
        will be returned if user enters no value.
        
        :param msg: String - message to display to user.
        :param default: String - default value to display to user. This 
            is returned if there is no input from the user.
        :param prompt: String - prompt to display to user awaiting input.
        :param mandatory: Boolean - Set True to re-prompt user if no value 
            entered.
        :rtype: String - the user input or default value, if any. A String, be
            it empty or not, is always returned.
        """
        
        input_string = ""
        if (default != None) and (len(default) > 0):
            input_string = ''.join([msg, ' (', default, ') ', prompt])
        else:
            input_string = ''.join([msg, ' ', prompt])

        gotInput = False
        while (gotInput is False):
            # strip leading and trailing whitespace from input. Assign 'default'
            # if no input returned.
            # 
            input = raw_input(input_string).strip()
            if (input == None) or (len(input) == 0):
                input = default
            
            gotInput = (len(input) > 0)
        
        return input

    @staticmethod
    def getInt(msg, default=0, prompt="> ", mandatory=True):
        """
        This function displays the argument message on the console,
        displays a prompt on the console, waits for user input, and 
        returns the input, or 'default' if no input. An integer value
        is always returned. If 'mandatory' argument is True (default) 
        then user will be re-prompted if no value entered. However, if
        'mandatory' is True and 'default' is supplied then 'default
        will be returned if user enters no value.
        
        :param msg: String - message to display to user.
        :param default: integer - default value to display to user. This 
            is returned if there is no input from the user.
        :param prompt: String - prompt to display to user awaiting input.
        :param mandatory: Boolean - Set True to re-prompt user if no value 
            entered.
        :rtype: integer - the user input or default value, if any. An integer 
            is always returned.
        """
        
        is_valid = False
        while (is_valid is False):
            try:
                val = 0
                if (default > 0):
                    val = int(Console.getString(msg, default=str(default), 
                                        prompt=prompt, mandatory=mandatory))
                else:
                    val = int(Console.getString(msg,
                                        prompt=prompt, mandatory=mandatory))
                is_valid = True
            except ValueError:
                Console.display("Bad numeric value '%s'" % val)
        
        return val
    
    @staticmethod
    def getUrl(msg, default="", prompt="> ", mandatory=True, scheme="http"):
        """
        This function displays the argument message on the console,
        displays a prompt on the console, waits for user input, and 
        returns the input, or 'default' if no input. A tuple value
        is always returned. If 'mandatory' argument is True (default) 
        then user will be re-prompted if no value entered. However, if
        'mandatory' is True and 'default' is supplied then 'default
        will be returned if user enters no value. Any entered value (but
        not 'default') will be checked for URL validity (ie. a non-Null
        scheme and hostname).
        
        :param msg: String - message to display to user.
        :param default: integer - default value to display to user. This 
            is returned if there is no input from the user.
        :param prompt: String - prompt to display to user awaiting input.
        :param mandatory: Boolean - Set True to re-prompt user if no value 
            entered.
        :param scheme: String - If supplied, the input URL scheme must match 
            this argument.
        :rtype: tuple - the user input or default value, if any. A tuple
            is always returned.
        """
        
        urlTuple = tuple()
        is_valid = False
        while (is_valid is False):
            urlString = Console.getString(msg, default=default, 
                                          prompt=prompt, mandatory=mandatory)
            urlTuple = urlparse.urlparse(urlString)
            
            # make sure input scheme matches argument scheme, 
            # but only if argument supplied.
            if (len(scheme) > 0):
                if ((len(urlTuple.scheme) == 0) or \
                    (urlTuple.scheme.lower() != scheme.lower())):
                    Console.display("Bad URL scheme. Must be '%s'" % scheme)
                elif ((urlTuple.netloc == None) or (len(urlTuple.netloc) == 0)):
                    Console.display("Bad URL hostname.")
                else:
                    is_valid = True
            else:
                is_valid = True

        return urlTuple
    
    @staticmethod
    def getUrlAsString(msg, default="", prompt="> ", mandatory=True, 
                       scheme="http"):
        """
        This function displays the argument message on the console,
        displays a prompt on the console, waits for user input, and 
        returns the input, or 'default' if no input. A String value
        is always returned. If 'mandatory' argument is True (default) 
        then user will be re-prompted if no value entered. However, if
        'mandatory' is True and 'default' is supplied then 'default
        will be returned if user enters no value. Any entered value (but
        not 'default') will be checked for URL validity (ie. a non-Null
        scheme and hostname).
        
        :param msg: String - message to display to user.
        :param default: integer - default value to display to user. This 
            is returned if there is no input from the user.
        :param prompt: String - prompt to display to user awaiting input.
        :param mandatory: Boolean - Set True to re-prompt user if no value 
            entered.
        :param scheme: String - If supplied, the input URL scheme must match 
            this argument.
        :rtype: String - the user input or default value, if any. A String
            is always returned.
        """
        
        urlTuple = Console.getUrl(msg, default, prompt, mandatory, scheme)
        return urlTuple.geturl()
    

class Configurables(object):
    """
    Internal class to store configuration settings.
    
    **Constants**:
    """
    
    def __init__(self):
        # Constructor.
        #
        
        self.proxyurl = None
        self.proxyuser = None
        self.proxypassword = None
        self.firewalldownloadsiteurl = None
        self.firewalladminsiteurl = None
        self.distributiondatastoreurl = None
        self.adminsiteurl = None
        self.nodeipaddress = None
        self.nodeportrangehigh = None
        self.nodeportrangelow = None
        self.usingproxy = None
        self.adminpassword_md5_digest = None
        self.firewall = None
        self.extra = None
        

class Distribution(object):
    """
    Internal class to store distribution related data.
    
    **Constants**:
    """
    
    def __init__(self):
        # Constructor.
        #
        
        self.exe_pathname = ""
        self.distribution_datastore_contents = None
        self.database_pathname = ""
        self.database_filename = ""
        self.node_ID = None    


class LoggingLevels(object):
    """
    Define extra logging levels.
    
    Note: Because datastore is a stand-alone package, this class is not used,
    rather, each module hard-codes the values as constants.
    
    **Constants**:
    
    .. data:: DEVEL
        
        DEVEL is a logging level between logging.DEBUG and logging.INFO.
            
    .. data:: TRACE
            
        TRACE is the lowest logging level. All log messages are dumped.
    """

    DEVEL = 15   # Data to support a developer writing a new task.
    TRACE = 5    # Lots of output to fix gnarly bugs  


class TCPServerHandler(SocketServer.BaseRequestHandler):
    """
    The RequestHandler class for our server.

    It is instantiated once per connection to the server, and must
    override the handle() method to implement communication to the
    client.
    
    **Constants**:
    """

    def __init__(self):
        # Constructor.
        #
        
        self.logger = logging.getLogger('yarra.runtime.node.TCPServerHandler')
        
    def handle(self):
        self.logger.log(logging.DEBUG, "")

        # We're only interested in processing the connection request, not
        # data on the connection.
        # 

class NodeServer(threading.Thread):
    """
    The purpose of this class is to act as a temporary server, accepting 
    connections on the configured IP address and port number. This is to test
    connectivity to configured node IP address and port number range. 

    **Constants**:
    """
    
    # Class constants
    #
    
    def __init__(self, ip_address, port_number):
        # Constructor.
        #
        
        threading.Thread.__init__(self)
        
        self.logger = logging.getLogger('yarra.runtime.node.NodeServer')
        self.ip_address = ip_address
        self.port_number = port_number
        
        try:
            self.server = SocketServer.TCPServer((self.ip_address, 
                                        self.port_number), TCPServerHandler)
        except BaseException as be:
            self.logger.exception("Cannot bind to IP address/port  (%s:%s)" % \
                                  (self.ip_address, self.port_number))
            raise Exception(be)
        
    def run(self):
        """
        This function is the mainline of the thread. It is called from start(). 
        """
        
        self.logger.log(logging.DEBUG, "Listening on (%s:%s)",
                        self.ip_address, self.port_number)
        
        self.server.handle_request()


class Install(object):
    """
    The purpose of this class is to control the installation of the node 
    software.

    **Constants**:
    """
    
    # Class constants
    #
    
    def __init__(self):
        # Constructor.
        #
        
        self.logger = logging.getLogger('yarra.runtime.node.Install')
    
    @staticmethod
    def download_url(download_url):
        """
        This function downloads the contents of the argument URL. 
        
        :param download_url: String - URL of file to download.
        :rtype: String - contents of the argument URL.
        """

        logging.log(logging.INFO, "Downloading URL (%s)" % download_url) 
        
        try:
            contents = None
            urlHandle = urllib2.urlopen(download_url)
        except IOError:
            logging.exception("Connection failure to (%s)" % download_url) 
            raise IOError("download_url(): Connection failure to (%s)" % \
                          download_url)
        else:
            contents = urlHandle.read()
            urlHandle.close()
            urlHandle = None
        
        return contents

    @staticmethod
    def get_first_item_of_serialised_resource(serialised_resource):
        """
        This function returns the first item in the argument serialised 
        resource. 
        
        :param serialised_resource: String - URL of file to download.
        :rtype: String - contents of the argument URL.
        """
        
        # the serialised resource will contain references to the 
        # following classes
        class ResourceDict(dict):
            pass
        class Data(object):
            def __init__(self, raw=None, encoded=None):
                self.data = encoded
        class Uri(str):
            pass
        
        #serialised_resource_xml_root = \
        #        xml.etree.ElementTree.XML(serialised_resource)
        try:
#            item = serialised_resource_xml_root.find('./content/item').text
            vars = eval(serialised_resource)
            item = vars['content']['content'].data
        except AttributeError:
            logging.exception("Unexpected Resource Structure")
            raise ResourceFormatException(
                "get_first_item_of_serialised_resource(): " + \
                "Unexpected resource Structure")
#            logging.exception("Unexpected XML Structure")
#            raise XMLFormatException(
#                "get_first_item_of_serialised_resource(): " + \
#                "Unexpected XML Structure")
        return item
    
    @staticmethod
    def create_python_module(module_name):
        """
        This function creates a new python module object for the argument 
        module_name, if the module_name does not already exist. If it does 
        already exist then it is returned.
        
        :param module_name: String - name of module.
        :rtype: module object
        """
        
        logging.log(logging.DEBUG, "creating module (%s)..." % module_name)  
        
        if not module_name in sys.modules:
            # create a new module
            #
            module = imp.new_module(module_name)
            
            # add module to the sys.modules as it will be used by other modules 
            #   
            sys.modules[module_name] = module
        else:
            module = sys.modules[module_name]
            
        return module

    @staticmethod
    def generate_node_ID(distribution):
        """
        This function generates a new node_ID - UUID Type 4 (Random) number. 
        
        :param distribution: Distribution - distribution configuration.
        :rtype: None
        """
        
        distribution.node_ID = str(uuid.uuid4())
        logging.log(logging.DEBUG, "new node_ID (%s)" % distribution.node_ID)
        
    @staticmethod
    def get_network_configurables(configurables, distribution):
        """
        This function prompts the user for network configuration items. Those 
        items that describe communication end-points will have their 
        connectivity checked. If connectivity fails the user will be 
        re-prompted.
        
        :param configurables: Configurables - user configuration items.
        :param distribution: Distribution - distribution configuration.
        :rtype: None
        """

        # External access (1=direct, 2=proxy, 3=firewall port forward)
        #
        if configurables.firewall is None:
            external_access = 0
            while (external_access < 1) or (external_access > 3):
                external_access = Console.getInt(
                    msg="Enter external access (1 = direct, 2 = proxy, " + \
                        "3 = firewall port forward)", 
                    default=1, mandatory=True)
        else:
            external_access = configurables.firewall
        
        if (external_access == 2):
            # get Proxy URL (mandatory) and credentials (optional)
            #
            urlHandle = None
            while (not urlHandle):
                if configurables.proxyurl is None:
                    configurables.proxyurl = Console.getUrl(
                        msg="Enter Proxy URL", mandatory=True, scheme="http")
                if configurables.proxyuser is None:
                    configurables.proxyuser = Console.getString(
                        msg="Enter Proxy user", mandatory=False)
                if configurables.proxypassword is None:
                    configurables.proxypassword = Console.getString(
                        msg="Enter Proxy password", mandatory=False)
                
                try:
                    url_parts = string.split(configurables.proxyurl.netloc, ":")
                    if (len(url_parts) == 1):
                        url_parts.append(DEFAULT_PROXY_PORT)
                    
                    proxy_info = {
                        'user' : configurables.proxyuser,
                        'pass' : configurables.proxypassword,
                        'host' : url_parts[0],
                        'port' : url_parts[1]
                    }
                    logging.log(logging.DEBUG, "Proxy details (%s)" % \
                                                proxy_info) 
                    
                    proxy_support = urllib2.ProxyHandler(
                        {"http":"http://%(user)s:%(pass)s@%(host)s:%(port)s" % \
                                                proxy_info})
                    opener = urllib2.build_opener(proxy_support, 
                                                urllib2.HTTPHandler)
                    urllib2.install_opener(opener)
                    
                    # download distribution file, just to test proxy connection.
                    #
                    urlHandle = urllib2.urlopen(TEST_SITE_URL)
                    
                except IOError:
                    Console.display("Connection failure through %s" % \
                                    configurables.proxyurl.geturl())
                else:
                    configurables.usingproxy = True
                    
        elif (external_access == 3):
            # get firewall port-forward URL for download site. 
            #
            if configurables.firewalldownloadsiteurl is None:
                str = ""
                while (str == ""):
                    configurables.firewalldownloadsiteurl = Console.getUrl(
                        msg="Enter firewall URL (to access download site)", 
                        mandatory=True, scheme="http") 
                    try:
                        # Here we open a connection to the download site home 
                        # page (ie. GET /) to prove we can connect through the 
                        # firewall.
                        str = Install.download_url(
                                configurables.firewalldownloadsiteurl.geturl())
                    except IOError:
                        Console.display("Connection failure to %s" % \
                                configurables.firewalldownloadsiteurl.geturl())
            
            # get firewall port-forward URL of admin site.
            if configurables.firewalladminsiteurl is None:
                str = ""
                while (str == ""):
                    configurables.firewalladminsiteurl = Console.getUrl(
                        msg="Enter firewall URL (to access admin site)", 
                        mandatory=True, scheme="http")
                    try:
                        # Here we open a connection to the admin site home page 
                        # (ie. GET /) to prove we can connect through the 
                        # firewall.
                        str = Install.download_url(
                                    configurables.firewalladminsiteurl.geturl())
                    except IOError:
                        Console.display("Connection failure to %s" % \
                                configurables.firewalladminsiteurl.geturl())
            
                    # TODO: The format of the download and admin site URLs for 
                    # port-forwarding will (probably) contain the firewall IP 
                    # address and a firewall port number. The firewall will 
                    # map requests to an external IP address/port. Should we 
                    # therefore also capture the hostname for each of the  
                    # download and admin site. Its likely that these sites 
                    # will need the hostname in the 'Host:' HTTP header field.
 
        
        # get URL for distribution datastore. If the user has entered 
        # port-forwarding URLs then this URL must include the host:port of the 
        # firewall that is mapped to the host:port of the download site.
        #
        # Allow user to load from current machine. Therefore don't restrict 
        # user to specific scheme. (ie. allow 'http:', 'ftp:', 'file:', etc
        #
        if configurables.distributiondatastoreurl is None:
            while (distribution.distribution_datastore_contents == None):
                configurables.distributiondatastoreurl = Console.getUrl(
                    default=DEFAULT_DOWNLOAD_SITE_DISTRIBUTION_DATASTORE_URL, 
                    msg="Enter distribution datastore URL", 
                                    mandatory=True, scheme="")
                try:
                    distribution.distribution_datastore_contents = \
                        Install.download_url(
                            configurables.distributiondatastoreurl.geturl())
                except IOError:
                    Console.display("Connection failure to %s" % \
                                    configurables.distributiondatastoreurl)
        else:
            distribution.distribution_datastore_contents = \
                Install.download_url(configurables.distributiondatastoreurl)
        
        # get URL of admin site. If the user has entered port-forwarding URLs 
        # then this URL must include the host:port of the firewall that is 
        # mapped to the host:port of the admin site.
        if configurables.adminsiteurl is None:
            str = ""
            while (str == ""):
                configurables.adminsiteurl = Console.getUrlAsString(
                    default=DEFAULT_ADMIN_SITE_URL,
                    msg="Enter central node administration site URL", 
                    mandatory=True, scheme="http")
                try:
                    str = Install.download_url(configurables.adminsiteurl)
                except IOError:
                    Console.display("Connection failure to %s" % \
                                    configurables.adminsiteurl)
        
        # get the node IP address. 
        #
        if configurables.nodeipaddress is None:
            is_valid = False
            while (is_valid is False):
                configurables.nodeipaddress = Console.getString(
                    default=DEFAULT_NODE_IP,
                    msg="Enter node IP address", mandatory=True)
                if configurables.nodeportrangelow is None:
                    configurables.nodeportrangelow = Console.getInt(
                        default=DEFAULT_PORT_RANGE_START,
                        msg="Enter node(s) port number range start", 
                                    mandatory=True)
                if configurables.nodeportrangehigh is None:
                    configurables.nodeportrangehigh = Console.getInt(
                        default=DEFAULT_PORT_RANGE_END,
                        msg="Enter node(s) port number range end",
                                    mandatory=True)
                
                # FIXME: Do we have to connect() to the IP address/port# 
                # or just bind() to it?
                
                is_ports_valid = True
                for port in range(configurables.nodeportrangelow, 
                                  configurables.nodeportrangehigh + 1):
                    nodeServer = None
                    try:
                        nodeServer = NodeServer(configurables.nodeipaddress, 
                                                port)
                    except BaseException as be:
                        is_ports_valid = False
                    
                    
                    # TODO: If we wish to connect() to the port then 
                    # un-comment the following code.
                    #
#                if (nodeServer != None):
#                    nodeServer.start()
#                    conn = socket.create_connection((
#                        configurables.nodeipaddress, port), timeout=2)
#                    nodeServer.join()
                
                is_valid = is_ports_valid
            

class DataStore(object):
    """
    The purpose of this class is to control the creation of the database.

    **Constants**:
    """
    
    # Class constants
    #

    def __init__(self):
        # Constructor.
        #
        
        self.logger = logging.getLogger('yarra.runtime.node.DataStore')
    
    @staticmethod
    def exists(database_filename):
        """
        This function checks if the database exists.
        
        :param database_filename: String - pathname of database file.
        :rtype: True - database exists
        :rtype: False - database does not exist
        """
        
        return os.path.exists(database_filename)

    @staticmethod
    def delete(database_filename):
        """
        This function deletes the argument database filename.
        
        :param database_filename: String - pathname of database file.
        :rtype: None.
        """
        
        if (DataStore.exists(database_filename)):
            os.remove(database_filename)
    
    @staticmethod
    def write(database_filename, contents):
        """
        This function writes the argument contents to the argument
        file descriptor.
        
        :param database_filename: String - pathname of database file.
        :param contents: String - contents to write to datastore.
        :rtype: None.
        """

        file = open(database_filename, "wb+")
        file.write(contents)
        file.close()

class MetaLoader(object):
    """
    The purpose of this class is to play the role of the loader in the 
    Importer Protocol.
    
    It duplicates some SQL functionality from datastore.database. 

    **Constants**:
    
    :param conn: DataStore connection
    :param resource_id: the id of the resouce that contains the module code. 
    :param metadata_key: the key to indicate whether the module path is a 
        package or module.
    :param init_file: boolean to indicate if the file we are loading is the init file.        
    """
    
    # Class constants
    #

    def __init__(self, conn, resource_id, metadata_key, init_file):
        # Constructor.
        #
        
        self.logger = logging.getLogger('yarra.runtime.node.MetaLoader')
        
        self._conn = conn
        self._resource_id = resource_id
        self._metadata_key = metadata_key
        self._init_file = init_file
        
    def load_module(self, module_path):
        """
        This function searches the datastore for the module with the argument
        module path and returns the module.
        
        :param module_path: String - pathname of the module.
        :rtype: module.
        """
                
        id = str(self._resource_id)
        cursor = self._conn.cursor()     
        cursor.execute("""
                        SELECT r.id as RESOURCE_ID, 
                            r.object as RESOURCE_OBJECT
                        FROM RESOURCE r
                        WHERE r.id = (?)
                        """, [id])
        
        # only one row should return
        resource_row = cursor.fetchone()

        serialised_resource = zlib.decompress(
                            resource_row['RESOURCE_OBJECT'])
        cursor.close()
        
        module_source_base64 = \
          Install.get_first_item_of_serialised_resource(serialised_resource)
        
        # we must decode the module code as it was encoded before being 
        # written to the datastore.
        module_source = zlib.decompress(
                            base64.b64decode(module_source_base64))
        
        # reuse the same module if it is already there. See PEP 302 for 
        # explanation
        if (module_path in sys.modules):
            module = sys.modules[module_path]
        else:
            module = Install.create_python_module(module_path)
            
            # stuff you have to set for modules see PEP 302
            module.__file__ = "<%s>" % self.__class__.__name__
            module.__loader = self 
        
        if self._init_file:
            # stuff you have to set for packages see PEP 302
            module.__path__ = []
        try:
            c = compile(module_source, "dyn:%s" % \
                        module_path, 'exec')
            # execute module's code in the module's global name space
            # Note that the parser only accepts the Unix-style end of line 
            # convention. If you are reading the code from a file, make sure 
            # to use universal newline mode to convert Windows or Mac-style 
            # newlines.
            # http://docs.python.org/reference/simple_stmts.html#exec
            #    if chr(0) in module_source:
            #        print("qqqqqqqqqqqqqqqqqqqq", module_source_base64)
            self.logger.log(LoggingLevels.TRACE, "Executing Code into (%s)", 
                            module_path)                      
            exec c in module.__dict__
        except:
            self.logger.exception("Error during module exec (%s)" % \
                        sys.exc_info()[1])
        
        self.logger.log(LoggingLevels.TRACE, "Module (%s) loaded", 
                        module_path)
                
        return sys.modules[module_path]
    
    def __cmp__(self, other):
        if (self._resource_id == other._resource_id) and \
            (self._metadata_key == other._metadata_key):
            return 0
        else:
            return -1
    
class MetaFinder(object):
    """
    The purpose of this class is to play the role of the finder in the 
    Importer Protocol.

    It duplicates some SQL functionality from datastore.database. 

    **Constants**:
    
    :param conn: DataStore connection
    """
    
    # Class constants
    #
    
    def __init__(self, conn):
        # Constructor.
        #
        
        self.logger = logging.getLogger('yarra.runtime.node.MetaFinder')
        self.conn = conn
    
    def find_module(self, fullname, path=None):
        """
        This function returns a MetaLoader instance if the module is found
        in the datastore, otherwise None. When None is returned, the import is
        of the module is delegated to the built in import function.
        
        :param fullname: String - full name of the module to import from the
            datastore.
        :rtype: MetaLoader.
        """
        logging_level = LoggingLevels.TRACE
        self.logger.log(logging_level, "Looking for (%s)...", fullname)
        self.logger.log(logging_level, 
                        "find_module(): key (%s), value (%s)",
            RESOURCE_KEY_URI_PACKAGE_MODULE, fullname)
        
        try:
#            conn = sqlite3.connect(self._database_filename)
#            conn.row_factory = sqlite3.Row
            cursor = self.conn.cursor()
            cursor.execute("""
                            SELECT resource_id as RESOURCE_ID, 
                                    key as RESOURCE_METADATA_KEY
                            FROM RESOURCEMETADATA
                            WHERE 
                            (key = (?) AND value = (?))
                            """,
                            [RESOURCE_KEY_URI_PACKAGE_MODULE, fullname])
            row = cursor.fetchone()
            cursor.close()             
            #self.conn.commit()
            if (row):
                resource_id = row['RESOURCE_ID']
                resource_metadata_key = row['RESOURCE_METADATA_KEY']
                cursor = self.conn.cursor()                
                cursor.execute("""
                                SELECT value as FILE_NAME
                                FROM RESOURCEMETADATA 
                                WHERE 
                                (key = (?) AND resource_id = (?))
                                """,
                                [RESOURCE_KEY_URI_FILE, resource_id])

                row = cursor.fetchone()
                file_name = row['FILE_NAME']
                if file_name == '__init__.py':
                    init_file = True
                else:
                    init_file = False                  
                self.logger.log(logging_level, 
                                "Found (%s). Now loading from DB", fullname)
                return MetaLoader(self.conn, 
                                  resource_id, 
                                  resource_metadata_key,
                                  init_file)
            else:
                self.logger.log(logging_level, 
                                "Module (%s) not found in datastore.", fullname)
                return None
        except Exception:
            self.logger.exception("Failure retrieving module (%s)", fullname)

class ResourceFormatException(Exception):
    """
    This class represents an exception for bad resource.
    
    **Constants**:
    """
    
    pass

#class XMLFormatException(Exception):
#    """This class represents an exception for bad XML."""
#    pass

#def configure_python_import(distribution, disable_import_hookin=False):
#    """This function configures Python to import modules from the 
#    datastore. By default we want to load imported modules from the 
#    datastore, however the caller can disable this by specifying 
#    the argument 'disable_import_hookin' = True.
#    
#    :param distribution: Distribution - distribution configuration.
#    """
#
#    if (disable_import_hookin is False):
#        # register the import hook.
#        #
#        sys.meta_path = [MetaFinder(os.path.join(distribution.database_pathname, 
#                                            distribution.database_filename))]

def keyboard_interrupt_handler(signum, frame):
    """
    This function is called when the user interrupts the process with a 
    Ctrl-C from the command line. 
    
    :param signum: int - the signal number.
    :param frame: Frame Object - the current stack frame.
    """
    
    logging.log(logging.ERROR, "Signal (%d) caught. Exiting." % signum)
    sys.exit(0)


if __name__ == '__main__':
    
    # Add new logging levels.
    logging.addLevelName(LoggingLevels.DEVEL, 'DEVEL')
    logging.addLevelName(LoggingLevels.TRACE, 'TRACE')    
    
    # Set the format of all log messages.
    streamHandler = logging.StreamHandler()
    formatter = logging.Formatter(
                    "[%(asctime)s %(levelname)-8s %(threadName)s %(name)s " + \
                    "%(module)s:%(lineno)d::%(funcName)s] %(message)s")
    streamHandler.setFormatter(formatter)
    logging.root.addHandler(streamHandler)
    handler = logging.handlers.RotatingFileHandler(LOG_FILE_NAME, maxBytes=TEN_MEGABYTES_IN_BYTES, backupCount=5)
    handler.setFormatter(formatter)
    logging.root.addHandler(handler)
    
    # TODO: Re-instate Windows Logging.   
    # Also send logging to Windows event logger.
    # NOTE: The Windows event log handler has default knowledge of the 'logging' 
    #       module's log levels and how they relate to Windows Event Log levels.
    #       We must add knowledge for our own levels.
    #
#    ntEventLogHandler = NTEventLogHandler("HAF eDA")
#    ntEventLogHandler.setFormatter(formatter)
#    ntEventLogHandler.typemap = {
#        logging.DEBUG   : win32evtlog.EVENTLOG_INFORMATION_TYPE,
#        logging.INFO    : win32evtlog.EVENTLOG_INFORMATION_TYPE,
#        logging.WARN    : win32evtlog.EVENTLOG_WARNING_TYPE,
#        logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
#        logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
#        LoggingLevels.DEVEL : win32evtlog.EVENTLOG_INFORMATION_TYPE,
#        LoggingLevels.TRACE : win32evtlog.EVENTLOG_INFORMATION_TYPE
#        }
#    logging.root.addHandler(ntEventLogHandler)
    
    # Get logger for this module.
    logger = logging.getLogger('yarra.runtime.node')
    
    # process the command line.
    usage = """
    %s [<options>] <command> [<datastore file>]
    
    To initialise the node (first run):
        <path>/node.exe initialise
    
    To install the node as a service:
        <path>/node.exe install <datastore name>

    To remove the node from Windows Services:
        <path>/node.exe remove <datastore name>

    To run the node as a service is:
        <path>/node.exe start-service <datastore name>

    To run the node from a shell is:
        <path>/node.exe start <datastore name>
    """  % sys.argv[0]
    parser = OptionParser(usage=usage)
    parser.add_option("--firewall", dest="firewall",
                      action="store", type="int",
                      default=None,
                      help="(Initialise) Firewall access (1 = direct, " + \
                           "2 = proxy, 3 = firewall port forward).")
    parser.add_option("--datastore", "--distribution-datastore-url", 
                      dest="distributiondatastoreurl",
                      action="store", type="string",
                      default=None,
                      help="(Initialise) Distribution data store URL.")
    parser.add_option("--proxy-url",
                      dest="proxyurl",
                      action="store", type="string",
                      default=None,
                      help="(Initialise) Proxy URL.")
    parser.add_option("--proxy-user",
                      dest="proxyuser",
                      action="store", type="string",
                      default=None,
                      help="(Initialise) Proxy User.")
    parser.add_option("--proxy-password",
                      dest="proxypassword",
                      action="store", type="string",
                      default=None,
                      help="(Initialise) Proxy User's password.")
    parser.add_option("--firewall-download-site-url",
                      dest="firewalldownloadsiteurl",
                      action="store", type="string",
                      default=None,
                      help="(Initialise) firewall download site url.")
    parser.add_option("--firewall-admin-site-url",
                      dest="firewalladminsiteurl",
                      action="store", type="string",
                      default=None,
                      help="(Initialise) firewall admin site url.")
    parser.add_option("--admin-site-url",
                      dest="adminsiteurl",
                      action="store", type="string",
                      default=None,
                      help="(Initialise) Central node admin site URL.")
    parser.add_option("--ip", "--node-ip", dest="nodeipaddress",
                      action="store", type="string",
                      default=None,
                      help="(Initialise) IP address to bind to the node.")
    parser.add_option("--start-port", dest="nodeportrangelow", 
                      action="store", type="int", 
                      default=None,
                      help="(Initialise) First of the block of port " + \
                        "numbers for use by the node.")
    parser.add_option("--end-port", dest="nodeportrangehigh", 
                      action="store", type="int", 
                      default=None,
                      help="(Initialise) Last of the block of port " + \
                        "numbers for use by the node.")
    parser.add_option("--test-node-id",
                      dest="test_node_id",
                      action="store", type="string",
                      default=None,
                      help="(Initialise) [testing] set node id to a " + \
                      "particular value.")
    parser.add_option("--log-level", dest="log_level", 
                      action="store", type="int",
                      default=logging.INFO, 
                      help="Set the log level (0=everything to 50=critical)")
    parser.add_option("--extra", dest="extra", 
                      action="append", type="string",
                      help="Extra options passed to the root Resource")
    parser.add_option("--dev", dest="dev", 
                      action="store_true",
                      help="Start in dev mode.")
    
    (options, args) = parser.parse_args()
    options.script = os.path.split(sys.argv[0])[1]
    
    # Set logging level
    logging.root.setLevel(int(options.log_level))

    if len(args) == 0:
        parser.error("No command specified.")
    command = args[0]
    
    # GET rid of this soon #############################################################
    distribution = Distribution()

    if command == 'initialise':
        logger.log(logging.INFO, "Initialising new node...")
        
        configurables = Configurables()
        # save any options set from the command line.
        for key in configurables.__dict__.keys():
            if key in options.__dict__:
                configurables.__dict__[key] = options.__dict__[key]
    
        # Generate new node_ID
        if len(options.test_node_id) > 0:
            logger.log(logging.INFO, "Generating fixed test Node ID...") 
            distribution.node_ID = options.test_node_id
        else:
            logger.log(logging.INFO, "Generating new Node ID...") 
            Install.generate_node_ID(distribution)
        
        logger.log(logging.INFO, "Get node configuration from user...") 
        Install.get_network_configurables(configurables, distribution)
        
        # Download the distribution database...
        logger.log(logging.INFO, "Installing distribution...") 

        # Write distribution datastore to disk with filename:
        # - <node-ID>.datastore
        distribution.database_filename = str(distribution.node_ID) + \
                                            ".datastore"
        datastore_name = os.path.join(distribution.database_pathname, 
                                distribution.database_filename)
        DataStore.write(datastore_name, 
                        distribution.distribution_datastore_contents)
        Console.display("Datastore name is %s" % datastore_name)
        command = 'initialise2'
    else:
        if len(args) == 1:
            parser.error("Datastore name missing for command: %s" % command)
        datastore_name = args[1]

    # MetaFinder needs to use the datastore to load code, including Datastore
    # and Runtime. DataStore is initialised with a connection and Runtime
    # with the datastore.
    conn = sqlite3.connect(datastore_name, check_same_thread=False)
    conn.row_factory = sqlite3.Row
    
    if not options.dev:
        sys.meta_path = [MetaFinder(conn)]
    from datastore.database import DataStore
    datastore = DataStore(filename=datastore_name, conn=conn)
    #datastore.lock()
    from runtime.rt import Runtime
    runtime = Runtime(datastore)

    if command == 'initialise2':
        logger.log(logging.INFO, "Updating node configuration after init.")  
        runtime.install(configurables, distribution)
 
    elif command == 'install':
        logger.log(logging.INFO, "Installing node as a Windows Service...")  
        runtime.install_service(distribution)

    elif command == 'remove':
        logger.log(logging.INFO, "Removing node from Windows Services...") 
        runtime.remove_service()

    elif command == 'start':
        logger.log(logging.INFO, 
            "Starting the yarra environment as a sub-process...")
        logger.log(LoggingLevels.TRACE, 
            "exe: %s prog: %s", sys.executable, sys.argv[0])
        
        def wrap(s):
            if sys.platform == 'win32': return '"%s"' % s
            else: return s

        # unlock the datastore so the subprocess can use it
        #datastore.unlock()
        
        # build the subprocess args list and then start as a sub-process          
        sp = []
        sp.append(sys.executable)
        if not sys.executable.endswith('node.exe'):
            sp.append(wrap(sys.argv[0]))
        sp.append('--log-level')
        sp.append(str(options.log_level))
        sp.append('start-sub-process')
        sp.append(datastore_name)
        retcode = 10
        while retcode == 10:
            retcode = subprocess.call(sp)

    elif command == 'start-sub-process':
        logger.log(logging.INFO, "Starting the yarra environment...")
        runtime.start_process(distribution)
        
    elif command == 'start-service':
        logger.log(logging.INFO, "Starting the yarra environment...") 
        runtime.start_service(distribution)
        
    else:
        parser.error("Invalid command (%s)." % command)
        
    logger.log(logging.DEBUG, "Done.")
